<!DOCTYPE html>
<html lang=en>
<head>
    <!-- so meta -->
    <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="HandheldFriendly" content="True">
    <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1" />
    <meta name="description" content="故事是這様的，我們有一批網站搬到新主機出現詭異現象：每隔一段時間某些 IIS AppPool Process 會吃滿 25% CPU 使用量，在 4 核機器這象徵有一條 Thread 陷入無窮迴圈吃光一個 CPU Core 的時間。有時也會出現多個 AppPool 同時發難，每個 Process 吃 25%，把整體 CPU 使用率逼上 50％、75％，甚至 100％。出問題時，該 AppPool">
<meta property="og:type" content="article">
<meta property="og:title" content="windbg调试">
<meta property="og:url" content="http://yoursite.com/2019/03/15/windbg%E8%B0%83%E8%AF%95/index.html">
<meta property="og:site_name" content="TmoonSite">
<meta property="og:description" content="故事是這様的，我們有一批網站搬到新主機出現詭異現象：每隔一段時間某些 IIS AppPool Process 會吃滿 25% CPU 使用量，在 4 核機器這象徵有一條 Thread 陷入無窮迴圈吃光一個 CPU Core 的時間。有時也會出現多個 AppPool 同時發難，每個 Process 吃 25%，把整體 CPU 使用率逼上 50％、75％，甚至 100％。出問題時，該 AppPool">
<meta property="og:locale" content="en_US">
<meta property="article:published_time" content="2019-03-15T02:16:52.000Z">
<meta property="article:modified_time" content="2020-01-07T10:48:14.738Z">
<meta property="article:author" content="Tmoonlight">
<meta name="twitter:card" content="summary">
    
    
        
          
              <link rel="shortcut icon" href="/images/favicon.ico">
          
        
        
          
            <link rel="icon" type="image/png" href="/images/logo2.gif" sizes="192x192">
          
        
        
          
            <link rel="apple-touch-icon" sizes="180x180" href="/images/logo2.gif">
          
        
    
    <!-- title -->
    <title>windbg调试</title>
    <!-- styles -->
    
<link rel="stylesheet" href="/css/style.css">

    <!-- persian styles -->
    
      
<link rel="stylesheet" href="/css/rtl.css">

    
    <!-- rss -->
    
    
<meta name="generator" content="Hexo 4.2.0"></head>

<body class="max-width mx-auto px3 ltr">
    
      <div id="header-post">
  <a id="menu-icon" href="#"><i class="fas fa-bars fa-lg"></i></a>
  <a id="menu-icon-tablet" href="#"><i class="fas fa-bars fa-lg"></i></a>
  <a id="top-icon-tablet" href="#" onclick="$('html, body').animate({ scrollTop: 0 }, 'fast');" style="display:none;"><i class="fas fa-chevron-up fa-lg"></i></a>
  <span id="menu">
    <span id="nav">
      <ul>
         
          <li><a href="/">Home</a></li>
         
          <li><a href="/about/">About</a></li>
         
          <li><a href="/archives/">Writing</a></li>
         
          <li><a href="/projects_url">Projects</a></li>
        
      </ul>
    </span>
    <br/>
    <span id="actions">
      <ul>
        
        <li><a class="icon" href="/2019/03/21/%E5%85%A5%E5%8F%A3%E7%A8%8B%E5%BA%8F%E6%89%BE%E4%B8%8D%E5%88%B0%E5%BC%95%E7%94%A8%E7%9A%84%E7%AC%AC%E4%B8%89%E6%96%B9%E5%BA%93/"><i class="fas fa-chevron-left" aria-hidden="true" onmouseover="$('#i-prev').toggle();" onmouseout="$('#i-prev').toggle();"></i></a></li>
        
        
        <li><a class="icon" href="/2019/03/10/DerMenschkanntunwaserwill;erkannabernichtwollenwaserwill/"><i class="fas fa-chevron-right" aria-hidden="true" onmouseover="$('#i-next').toggle();" onmouseout="$('#i-next').toggle();"></i></a></li>
        
        <li><a class="icon" href="#" onclick="$('html, body').animate({ scrollTop: 0 }, 'fast');"><i class="fas fa-chevron-up" aria-hidden="true" onmouseover="$('#i-top').toggle();" onmouseout="$('#i-top').toggle();"></i></a></li>
        <li><a class="icon" href="#"><i class="fas fa-share-alt" aria-hidden="true" onmouseover="$('#i-share').toggle();" onmouseout="$('#i-share').toggle();" onclick="$('#share').toggle();return false;"></i></a></li>
      </ul>
      <span id="i-prev" class="info" style="display:none;">Previous post</span>
      <span id="i-next" class="info" style="display:none;">Next post</span>
      <span id="i-top" class="info" style="display:none;">Back to top</span>
      <span id="i-share" class="info" style="display:none;">Share post</span>
    </span>
    <br/>
    <div id="share" style="display: none">
      <ul>
  <li><a class="icon" href="http://www.facebook.com/sharer.php?u=http://yoursite.com/2019/03/15/windbg%E8%B0%83%E8%AF%95/" target="_blank" rel="noopener"><i class="fab fa-facebook " aria-hidden="true"></i></a></li>
  <li><a class="icon" href="https://twitter.com/share?url=http://yoursite.com/2019/03/15/windbg%E8%B0%83%E8%AF%95/&text=windbg调试" target="_blank" rel="noopener"><i class="fab fa-twitter " aria-hidden="true"></i></a></li>
  <li><a class="icon" href="http://www.linkedin.com/shareArticle?url=http://yoursite.com/2019/03/15/windbg%E8%B0%83%E8%AF%95/&title=windbg调试" target="_blank" rel="noopener"><i class="fab fa-linkedin " aria-hidden="true"></i></a></li>
  <li><a class="icon" href="https://pinterest.com/pin/create/bookmarklet/?url=http://yoursite.com/2019/03/15/windbg%E8%B0%83%E8%AF%95/&is_video=false&description=windbg调试" target="_blank" rel="noopener"><i class="fab fa-pinterest " aria-hidden="true"></i></a></li>
  <li><a class="icon" href="mailto:?subject=windbg调试&body=Check out this article: http://yoursite.com/2019/03/15/windbg%E8%B0%83%E8%AF%95/"><i class="fas fa-envelope " aria-hidden="true"></i></a></li>
  <li><a class="icon" href="https://getpocket.com/save?url=http://yoursite.com/2019/03/15/windbg%E8%B0%83%E8%AF%95/&title=windbg调试" target="_blank" rel="noopener"><i class="fab fa-get-pocket " aria-hidden="true"></i></a></li>
  <li><a class="icon" href="http://reddit.com/submit?url=http://yoursite.com/2019/03/15/windbg%E8%B0%83%E8%AF%95/&title=windbg调试" target="_blank" rel="noopener"><i class="fab fa-reddit " aria-hidden="true"></i></a></li>
  <li><a class="icon" href="http://www.stumbleupon.com/submit?url=http://yoursite.com/2019/03/15/windbg%E8%B0%83%E8%AF%95/&title=windbg调试" target="_blank" rel="noopener"><i class="fab fa-stumbleupon " aria-hidden="true"></i></a></li>
  <li><a class="icon" href="http://digg.com/submit?url=http://yoursite.com/2019/03/15/windbg%E8%B0%83%E8%AF%95/&title=windbg调试" target="_blank" rel="noopener"><i class="fab fa-digg " aria-hidden="true"></i></a></li>
  <li><a class="icon" href="http://www.tumblr.com/share/link?url=http://yoursite.com/2019/03/15/windbg%E8%B0%83%E8%AF%95/&name=windbg调试&description=" target="_blank" rel="noopener"><i class="fab fa-tumblr " aria-hidden="true"></i></a></li>
  <li><a class="icon" href="https://news.ycombinator.com/submitlink?u=http://yoursite.com/2019/03/15/windbg%E8%B0%83%E8%AF%95/&t=windbg调试" target="_blank" rel="noopener"><i class="fab fa-hacker-news " aria-hidden="true"></i></a></li>
</ul>

    </div>
    <div id="toc">
      <ol class="toc"><li class="toc-item toc-level-1"><a class="toc-link" href="#null"><span class="toc-number">1.</span> <span class="toc-text"></span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#WinDbg-好威呀！"><span class="toc-number">1.0.0.1.</span> <span class="toc-text">WinDbg 好威呀！</span></a></li></ol></li></ol></li></ol></li></ol>
    </div>
  </span>
</div>

    
    <div class="content index py4">
        
        <article class="post" itemscope itemtype="http://schema.org/BlogPosting">
  <header>
    
    <h1 class="posttitle" itemprop="name headline">
        windbg调试
    </h1>



    <div class="meta">
      <span class="author" itemprop="author" itemscope itemtype="http://schema.org/Person">
        <span itemprop="name">TmoonSite</span>
      </span>
      
    <div class="postdate">
      
        <time datetime="2019-03-15T02:16:52.000Z" itemprop="datePublished">2019-03-15</time>
        
      
    </div>


      

      

    </div>
  </header>
  

  <div class="content" itemprop="articleBody">
    <h1 id=""><a href="#" class="headerlink" title=""></a></h1><p>故事是這様的，我們有一批網站搬到新主機出現詭異現象：每隔一段時間某些 IIS AppPool Process 會吃滿 25% CPU 使用量，在 4 核機器這象徵有一條 Thread 陷入無窮迴圈吃光一個 CPU Core 的時間。有時也會出現多個 AppPool 同時發難，每個 Process 吃 25%，把整體 CPU 使用率逼上 50％、75％，甚至 100％。出問題時，該 AppPool 網站仍能使用，但無法透過 IIS 管理回收 AppPool，只能用 TaskManager 砍掉 Process。砍掉 Process 後，系統會乖一陣子，但幾個小時或隔天又復發。</p>
<p>原本正常的程式移到新環境不穩定讓人心慌，很想知道吃光 CPU 的 AppPool 究竟在忙什麼？到底是哪支網頁是老鼠屎？我唯一想到的可用武器是強大但陌生的 WinDbg (Windows Debugger），爬文與一番摸索後居然真讓我成功抓出問題所在，興奮到想轉圈灑花。：P</p>
<p>以下就分享本次使用 WinDbg 找出 CPU 100％ 程式來源的經驗：</p>
<ol>
<li>使用 TaskManager 產生 Memory Dump 檔</li>
</ol>
<p>找出吃光 CPU 的 w3wp.exe Process，叫出右鍵選單執行「Create dump file」 產生記憶體傾印檔。</p>
<p>有件事要注意，要先釐清 AppPool 是 32 位元還是 64 位元，若為 32 位元需<a href="https://blogs.msdn.microsoft.com/tess/2010/09/29/capturing-memory-dumps-for-32-bit-processes-on-an-x64-machine/" target="_blank" rel="noopener">改用 32 位元版 TaskManager</a>，否則 Dump 檔解析出來的資訊會類似 wow64cpu!TurboDispatchJumpAddressEnd+0x598，看不到真實執行位置。32 位元版 TaskManager 位於 C:\Windows\SysWOW64\taskmgr.exe，執行後 Process 名稱應為「Task Manager (32bit)」。</p>
<p>若 CPU 是不定期飆高一陣子後恢復，SysInternals 有個超好用工具 <a href="https://blogs.iis.net/webtopics/using-procdump-exe-to-monitor-w3wp-exe-for-cpu-spikes" target="_blank" rel="noopener">procdump</a>，遇到 32 位元 Process 時會自動產生 32 位元 Dump 檔，還可以指定「當 CPU 高於 x% 並持續 y 秒時自動產生 Dump 檔」，可多加利用。</p>
<ol start="2">
<li>啟用 WinDbg 載入 Dump 檔案</li>
</ol>
<p>Windows Debugger 有 32/64 位元之分，請視要分析對象的位元版本選擇對應版本的 WinDbg。</p>
<p>啟動 WinDbg 後，使用 File / Open Crash Dump 開啟剛才取得的 w3wp.DMP。</p>
<p>註：Windows Debugger 下載安裝資訊請自行參考 <a href="https://msdn.microsoft.com/en-us/library/windows/hardware/ff551063(v=vs.85).aspx" target="_blank" rel="noopener">MSDN</a>，此處不多贅述。</p>
<ol start="3">
<li>WinDbg 的指令既多且雜，有興趣的朋友請參考<a href="http://windbg.info/doc/1-common-cmds.html" target="_blank" rel="noopener">文件</a>，此處記錄並說明我動用的指令：</li>
</ol>
<p>.sympath srv<em>X:\Symbol</em><a href="https://msdl.microsoft.com/download/symbols" target="_blank" rel="noopener">https://msdl.microsoft.com/download/symbols</a></p>
<p>分析過程需要 Symbol 檔，指定 WinDbg 自動由微軟網站下載，並 Cache 在 X:\Symbol 目錄避免重複下載</p>
<p>!sym noisy</p>
<p>指定顯示完整 Symbol 下載資訊</p>
<p>.cordll -ve -u  -l</p>
<p>自動載入 CLR 偵錯相關模組（分析來自其他台機器 Dump 檔時，這招好用）  </p>
<ol start="4">
<li>找出佔用 CPU 最多的 Thread，並將偵錯對象切換成該 Thread</li>
</ol>
<p>!runaway</p>
<p>找出每個 Thread 消耗的 CPU 時間，在本案例中，Thread 27 耗用超過一個小時，無疑是吃光 CPU 的兇手</p>
<p>~27s</p>
<p>將偵錯對象切換成 Thread 27</p>
<ol start="5">
<li>使用 !clrstack 列出 Thread 27 的 Callstack</li>
</ol>
<p>兇手現形！MasterPage 中某個共用元件使用 <a href="http://odp.net/" target="_blank" rel="noopener">ODP.NET</a> 讀取使用者基本資料，程式卡在 Oracle.DataAccess.Client.OpsSql.ExecuteReader()，有可能等待 Oracle 資料庫回應時陷入無窮迴圈並耗盡 CPU。</p>
<p>陸續分析了幾起案例的 Dump 檔，都能找到一條 Thread 明顯耗用大量 CPU，而 Callstack 問題都指向 Oracle.DataAccess.Client.OpsSql.ExecuteReader()，鎖定問題後推測是<a href="http://blog.darkthread.net/post-2012-04-10-ora-12571-under-nlb.aspx" target="_blank" rel="noopener">網路設備干擾 Oracle 連線</a>的問題（先前看過的症狀都是收到明確錯誤，像這樣 Hang 住的狀況是第一次遇到），避開後問題排除。</p>
<p>就這樣，靠著 WinDbg 漂亮偵破一個看似毫無頭緒的疑案。</p>
<p>呼口號時間：</p>
<h4 id="WinDbg-好威呀！"><a href="#WinDbg-好威呀！" class="headerlink" title="WinDbg 好威呀！"></a>WinDbg 好威呀！</h4>
  </div>
</article>



        
          <div id="footer-post-container">
  <div id="footer-post">

    <div id="nav-footer" style="display: none">
      <ul>
         
          <li><a href="/">Home</a></li>
         
          <li><a href="/about/">About</a></li>
         
          <li><a href="/archives/">Writing</a></li>
         
          <li><a href="/projects_url">Projects</a></li>
        
      </ul>
    </div>

    <div id="toc-footer" style="display: none">
      <ol class="toc"><li class="toc-item toc-level-1"><a class="toc-link" href="#null"><span class="toc-number">1.</span> <span class="toc-text"></span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#WinDbg-好威呀！"><span class="toc-number">1.0.0.1.</span> <span class="toc-text">WinDbg 好威呀！</span></a></li></ol></li></ol></li></ol></li></ol>
    </div>

    <div id="share-footer" style="display: none">
      <ul>
  <li><a class="icon" href="http://www.facebook.com/sharer.php?u=http://yoursite.com/2019/03/15/windbg%E8%B0%83%E8%AF%95/" target="_blank" rel="noopener"><i class="fab fa-facebook fa-lg" aria-hidden="true"></i></a></li>
  <li><a class="icon" href="https://twitter.com/share?url=http://yoursite.com/2019/03/15/windbg%E8%B0%83%E8%AF%95/&text=windbg调试" target="_blank" rel="noopener"><i class="fab fa-twitter fa-lg" aria-hidden="true"></i></a></li>
  <li><a class="icon" href="http://www.linkedin.com/shareArticle?url=http://yoursite.com/2019/03/15/windbg%E8%B0%83%E8%AF%95/&title=windbg调试" target="_blank" rel="noopener"><i class="fab fa-linkedin fa-lg" aria-hidden="true"></i></a></li>
  <li><a class="icon" href="https://pinterest.com/pin/create/bookmarklet/?url=http://yoursite.com/2019/03/15/windbg%E8%B0%83%E8%AF%95/&is_video=false&description=windbg调试" target="_blank" rel="noopener"><i class="fab fa-pinterest fa-lg" aria-hidden="true"></i></a></li>
  <li><a class="icon" href="mailto:?subject=windbg调试&body=Check out this article: http://yoursite.com/2019/03/15/windbg%E8%B0%83%E8%AF%95/"><i class="fas fa-envelope fa-lg" aria-hidden="true"></i></a></li>
  <li><a class="icon" href="https://getpocket.com/save?url=http://yoursite.com/2019/03/15/windbg%E8%B0%83%E8%AF%95/&title=windbg调试" target="_blank" rel="noopener"><i class="fab fa-get-pocket fa-lg" aria-hidden="true"></i></a></li>
  <li><a class="icon" href="http://reddit.com/submit?url=http://yoursite.com/2019/03/15/windbg%E8%B0%83%E8%AF%95/&title=windbg调试" target="_blank" rel="noopener"><i class="fab fa-reddit fa-lg" aria-hidden="true"></i></a></li>
  <li><a class="icon" href="http://www.stumbleupon.com/submit?url=http://yoursite.com/2019/03/15/windbg%E8%B0%83%E8%AF%95/&title=windbg调试" target="_blank" rel="noopener"><i class="fab fa-stumbleupon fa-lg" aria-hidden="true"></i></a></li>
  <li><a class="icon" href="http://digg.com/submit?url=http://yoursite.com/2019/03/15/windbg%E8%B0%83%E8%AF%95/&title=windbg调试" target="_blank" rel="noopener"><i class="fab fa-digg fa-lg" aria-hidden="true"></i></a></li>
  <li><a class="icon" href="http://www.tumblr.com/share/link?url=http://yoursite.com/2019/03/15/windbg%E8%B0%83%E8%AF%95/&name=windbg调试&description=" target="_blank" rel="noopener"><i class="fab fa-tumblr fa-lg" aria-hidden="true"></i></a></li>
  <li><a class="icon" href="https://news.ycombinator.com/submitlink?u=http://yoursite.com/2019/03/15/windbg%E8%B0%83%E8%AF%95/&t=windbg调试" target="_blank" rel="noopener"><i class="fab fa-hacker-news fa-lg" aria-hidden="true"></i></a></li>
</ul>

    </div>

    <div id="actions-footer">
        <a id="menu" class="icon" href="#" onclick="$('#nav-footer').toggle();return false;"><i class="fas fa-bars fa-lg" aria-hidden="true"></i> Menu</a>
        <a id="toc" class="icon" href="#" onclick="$('#toc-footer').toggle();return false;"><i class="fas fa-list fa-lg" aria-hidden="true"></i> TOC</a>
        <a id="share" class="icon" href="#" onclick="$('#share-footer').toggle();return false;"><i class="fas fa-share-alt fa-lg" aria-hidden="true"></i> Share</a>
        <a id="top" style="display:none" class="icon" href="#" onclick="$('html, body').animate({ scrollTop: 0 }, 'fast');"><i class="fas fa-chevron-up fa-lg" aria-hidden="true"></i> Top</a>
    </div>

  </div>
</div>

        
        <footer id="footer">
  <div class="footer-left">
    Copyright &copy; 2020 Tmoonlight
  </div>
  <div class="footer-right">
    <nav>
      <ul>
         
          <li><a href="/">Home</a></li>
         
          <li><a href="/about/">About</a></li>
         
          <li><a href="/archives/">Writing</a></li>
         
          <li><a href="/projects_url">Projects</a></li>
        
      </ul>
    </nav>
  </div>
</footer>

    </div>
    <!-- styles -->

<link rel="stylesheet" href="/lib/font-awesome/css/all.min.css">


<link rel="stylesheet" href="/lib/justified-gallery/css/justifiedGallery.min.css">


    <!-- jquery -->

<script src="/lib/jquery/jquery.min.js"></script>


<script src="/lib/justified-gallery/js/jquery.justifiedGallery.min.js"></script>

<!-- clipboard -->

  
<script src="/lib/clipboard/clipboard.min.js"></script>

  <script type="text/javascript">
  $(function() {
    // copy-btn HTML
    var btn = "<span class=\"btn-copy tooltipped tooltipped-sw\" aria-label=\"Copy to clipboard!\">";
    btn += '<i class="far fa-clone"></i>';
    btn += '</span>'; 
    // mount it!
    $(".highlight table").before(btn);
    var clip = new ClipboardJS('.btn-copy', {
      text: function(trigger) {
        return Array.from(trigger.nextElementSibling.querySelectorAll('.code')).reduce((str,it)=>str+it.innerText+'\n','')
      }
    });
    clip.on('success', function(e) {
      e.trigger.setAttribute('aria-label', "Copied!");
      e.clearSelection();
    })
  })
  </script>


<script src="/js/main.js"></script>

<!-- search -->

<!-- Google Analytics -->

<!-- Baidu Analytics -->

<!-- Disqus Comments -->


</body>
</html>
